package engine

import (
	"log"
	"my-crawler/single-task/fetcher"
)

func Run(reqs ...CrawlerRequest)  {
	for len(reqs) > 0 {
		//有请求就开始 Fetch -> parser -> 返回请求 -> 加入队列
		req := reqs[0]
		content,err := fetcher.FetchByGet(req.Url)
		if err != nil {
			log.Println(err) //不停止
			continue
		}
		parserResult := req.Parser(content)

		//返回的结果添加到reqs中
		reqs = append(reqs[1:],parserResult.Request...)

		//处理Item
	}

}
